// index.js
const fetchPage = require('./fetchPage');
const parseNews = require('./parser');
const { saveNews, saveImage } = require('./saveToDB');
const downloadImage = require('./downloadImage');
const isAllowed = require('./robotsCheck');
const delay = require('delay');

const url = 'https://news.qq.com/rain/a/20250513A04MTX00'; // 示例文章

(async () => {
  if (!(await isAllowed(url))) {
    console.log('该页面不允许被爬取');
    return;
  }

  const html = await fetchPage(url);
  if (!html) return;

  const news = parseNews(html, url);
  const newsId = await saveNews(news);

  for (const imgUrl of news.imageUrls) {
    const filename = await downloadImage(imgUrl, newsId);
    if (filename) await saveImage(newsId, filename);
    setTimeout(() => {

    }, 1000);; // 每张图片之间延迟1s
  }

  console.log('爬取完成');
})();
